{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "\n",
    "from path_explain.utils import set_up_environment\n",
    "from path_explain.path_explainer_tf import PathExplainerTF\n",
    "from embedding_explainer import EmbeddingExplainerTF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_up_environment(visible_devices='2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 10\n",
    "test_size  = (32, 64)\n",
    "data = np.random.randn(batch_size, *test_size).astype(np.float32)\n",
    "baseline = np.zeros((1, *test_size)).astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential()\n",
    "model.add(tf.keras.layers.Input(shape=test_size))\n",
    "model.add(tf.keras.layers.Conv1D(filters=16, kernel_size=3, strides=1, activation=tf.keras.activations.softplus))\n",
    "model.add(tf.keras.layers.Flatten())\n",
    "model.add(tf.keras.layers.Dense(units=32, activation=tf.keras.activations.softplus))\n",
    "model.add(tf.keras.layers.Dense(units=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = EmbeddingExplainerTF(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:00<00:00, 38.90it/s]\n"
     ]
    }
   ],
   "source": [
    "attributions = explainer.attributions(inputs=data,\n",
    "                                      baseline=baseline,\n",
    "                                      batch_size=50,\n",
    "                                      num_samples=200,\n",
    "                                      use_expectation=False,\n",
    "                                      output_indices=0,\n",
    "                                      verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:12<00:00,  1.27s/it]\n"
     ]
    }
   ],
   "source": [
    "interactions = explainer.interactions(inputs=data,\n",
    "                                      baseline=baseline,\n",
    "                                      batch_size=50,\n",
    "                                      num_samples=200,\n",
    "                                      use_expectation=False,\n",
    "                                      output_indices=0,\n",
    "                                      verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f7075b91d10>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAbU0lEQVR4nO3df5TcdX3v8edr1w3dAJpQ0ggha6jNpRcKQrsleK1KKxRKLyFF5Pe58VxsrrWce1tqNJhcQBQJpKCeK+falHIOKgqSwphWdAso51aPyc3SCVnDaSRwMMkEISpBr1nJJrzvHzMbJ8vs7sx8vzPznZnX45yc/X7n+5n5vs9m89pvPt/P9/NRRGBmZp2vp9UFmJlZczjwzcy6hAPfzKxLOPDNzLqEA9/MrEu8odUFTObYY4+NBQsWtLoMM7O28uSTT/44IuZUOpbZwF+wYAHDw8OtLsPMrK1I+uFkx9ylY2bWJRz4ZmZdwoFvZtYlUgl8SedL2iZpu6QVU7R7r6SQNJjGec3MrHqJA19SL3AX8CfAycAVkk6u0O5o4H8AG5Oe08zMapfGFf6ZwPaIeC4i9gP3AxdVaPcJ4Dbglymc08zMapRG4M8Ddpbt7yq9doik3wXmR8TXp/ogScskDUsa3rNnTwqlmZnZuIbftJXUA9wJ/M10bSNibUQMRsTgnDkVnxswM7M6pRH4BWB+2f4JpdfGHQ38DvCEpOeBs4D1vnFrZtZcaTxpuwlYKOlEikF/OXDl+MGIeAU4dnxf0hPAhyPCj9GaWVfL5QusGdrG7r2jHD+rn+XnncSSM+ZN/8Y6JQ78iDgg6VpgCOgF7omIrZJuBoYjYn3Sc5iZdZpcvsD1D40wOnYQgMLeUa5/aASgYaGfylw6EfEI8MiE126YpO3ZaZzTzKydrRnadijsx42OHWTN0LZsB76ZmU2vvAtnstXEd+8dbdj5HfhmZk2wKjfCfRt2TBr0446f1d+wGjyXjplZg+XyharCvr+vl+XnndSwOnyFb2bWYGuGtk0Z9oL2GKVjZmZTm6pfft6sfr674o+aUoe7dMzMGmyyfnlBQ7twJnLgm5k12PLzTqK/r/ew1wRcddZAQ7twJnKXjplZg42HejOfqq3EgW9m1gRLzpjX9ICfyF06ZmZdwoFvZtYlHPhmZl3CffhmZlNYlRvhKxt3cjCCXokrFs3nk0tObXVZdXHgm5lNYlVuhC9t2HFo/2DEof12DH0HvplZmWpmtPzKxp0OfDOzdjZxUZLJHIzppkHLJt+0NTMrqbQoSSW9UhOqSV8qgS/pfEnbJG2XtKLC8Q9KGpG0WdJ3JJ2cxnnNzNJU7eIjVyya3+BKGiNxl46kXuAu4FxgF7BJ0vqIeLqs2Zcj4vOl9ouBO4Hzk57bzCyJiYuIv6m/j72jY5O29ygdOBPYHhHPAUi6H7gIOBT4EfGzsvZHwrTrAJiZNVSlRcT7ekVfjxh77VcR1d/Xy60Xn9ryaRHSkEbgzwN2lu3vAhZNbCTpL4HrgBlAxcmfJS0DlgEMDAykUJqZWWWV+uvHDgazZ/Yxc8YbWjrJWaM0bZRORNwF3CXpSmAVsLRCm7XAWoDBwUH/L8DMGmay/vq9+8bI3/DHTa6mOdK4aVsAyu9gnFB6bTL3A0tSOK+ZWd0mW5SkkYuIt1oagb8JWCjpREkzgMuB9eUNJC0s2/1T4JkUzmtmVrdKi5I0ehHxVkvcpRMRByRdCwwBvcA9EbFV0s3AcESsB66VdA4wBrxMhe4cM7NmysqiJM2kyOgTY4ODgzE8PNzqMsysjUwcZtnpAV6JpCcjYrDSMU+tYGYdodIwy+sfGgHoutCfjKdWMLOOUGmY5ejYQdYMbWtRRdnjwDezjjDZMMtqp0voBu7SMbO2NLG/ftbMPl7e9/ppETp5mGWtHPhm1lZy+QIf/6eth4V7Ye8ofT2ir1eMHTx8WoROHmZZKwe+mbWNqearH3stmNXfx5FHdOa0CGlw4JtZ25huvvpXRsfYfGNnTouQBt+0NbO2Md0NWPfXT82Bb2ZtY6pAd3/99Bz4ZtY2Ks1/AzCrv69j5qxvJPfhm1km5PIFVj48wi/2/6qPfmZfD5+6+LRDQd6N89+kyYFvZi1TaYhluX1jr3HdVzcDHBb6Dvj6uEvHzFoily+wfN1Tk4b9uNcCT4+QEge+mbXEmqFthz0kNRVPj5AOB76ZtUQtIe7hlulwH76ZNcV0/fWT6REebpmSVAJf0vnAZymueHV3RKyecPw64APAAWAP8F8j4odpnNvMsq3eoAeY0Stuv+RtvkmbksSBL6kXuAs4F9gFbJK0PiKeLmuWBwYjYp+kvwBuBy5Lem4zy7ap5r6ZSl8PrHnf6Q76lKVxhX8msD0ingOQdD9wEXAo8CPi22XtNwBXp3BeM8uoeq/qeyWuWDSfTy45tUGVdbc0An8esLNsfxewaIr21wDfSOG8ZpZBq3IjfGnDjpre0yvx7K0XNKgiG9fUm7aSrgYGgXdPcnwZsAxgYGCgiZWZWRrqCXuAKxbNb0A1NlEawzILQPnf1gml1w4j6RxgJbA4Il6t9EERsTYiBiNicM6cOSmUZmbNkssXag77HsHVZw24C6dJ0rjC3wQslHQixaC/HLiyvIGkM4C/A86PiJdSOKeZZcxN67dW1W5Wf5/nrG+RxIEfEQckXQsMURyWeU9EbJV0MzAcEeuBNcBRwIOSAHZExOKk5zaz1lqVG+ErG3dyMKp7YravR9y0+JQGV2WTSaUPPyIeAR6Z8NoNZdvnpHEeM8uGXL7Axx7awr6x16p+T3GopcfUt5KftDWzql3199/ju8/+tOb3veOtx3Dfn7+9ARVZLRz4ZjatXL7Ahx98igOvVdd1U843ZbPDgW9mU6pnqKXAi5NkkAPfzCZVT9j7ij67HPhm9jqrciPct2EHtXTgSHDVIod9ljnwzeyQXL7AR9Y9xf4qFyYBz2jZThz4ZgbUNwJn4W8cyaPXnd2Ygix1XvHKzFiVG6k57N/x1mMc9m3GV/hmXWpVboQvb9xBrSMtBVzlG7NtyYFv1oXqHWr56cu8KEk7c5eOWZepd1ZLh3378xW+WZeoZ/4bgNkz+7jxwlMc9h3AgW/WBWrtwvFSg53JgW/WoXL5AmuGtlHYO1rT+z7jrpuO5cA360D1PCkLxaGWDvvO5cA36zD1jMDpEVzpaRE6ngPfrEPUe1Xvyc66RyrDMiWdL2mbpO2SVlQ4/i5J/ybpgKRL0jinmRXl8gVO/p/f4Es1hv2RM3r5zGWnO+y7SOIrfEm9wF3AucAuYJOk9RHxdFmzHcD7gQ8nPZ+Z/Uo989/M8zz1XSuNLp0zge0R8RyApPuBi4BDgR8Rz5eO1TYA2MwmVev8NzP7enj6E3/SwIos69II/HnAzrL9XcCiej5I0jJgGcDAwEDyysw6TL1DLXt7xKcuPq1BVVm7yNRN24hYC6wFGBwcrH3xTLMOlcsXWP7gZmp8SBYo9tXf8menugvHUgn8AjC/bP+E0mtmloJ6hllCsQvnUxef5qC3Q9II/E3AQkknUgz6y4ErU/hcs65Xz0RnM3rFD265oEEVWTtLHPgRcUDStcAQ0AvcExFbJd0MDEfEekm/DzwMzAYulPTxiDgl6bnNOlG9/fRQfIDq9kve1oCqrBOk0ocfEY8Aj0x47Yay7U0Uu3rMbAq5fIHrHthMPcPZZvX3cdNiz2ppk8vUTVuzbvdXD2yu+T0eV2/VcuCbZUA9D1C9463HcN+fv71BFVkncuCbtVAuX+Aj655i/8HqRyG768bq5cA3a5F6hlv6qt6ScOCbNVm94+p7hcPeEnHgmzVJvUEPxeGWd1x6esoVWbdx4Js1QT03Zcd5EXFLiwPfrIFy+QIrHx7hF/sP1vQ+99VbIzjwzRqgntE347wClTWKA98sZefe+QTPvPSLmt8n4CqHvTWQA98sJbl8gb/56mbquKj3Vb01hQPfLAX13pTtLY2+8Q1ZawYHvlkC9Qa9gE9f5qC35nLgm9XptBu/yc9erW30DRSXG7zjfW9z2FvTOfDNauQx9dauHPhmVcrlC/z1A5upZ7Fl35S1LHDgm1Wh3mkRHPSWJakEvqTzgc9SXOLw7ohYPeH4EcAXgN8DfgJcFhHPp3Fus0bJ5Qt89B+38OqBetafcthb9iQOfEm9wF3AucAuYJOk9RHxdFmza4CXI+K3JF0O3AZclvTcZo2SyxfqWn0KoAe40yNwLIPSuMI/E9geEc8BSLofuAgoD/yLgJtK2+uAz0lSRNTTHWrWUPV23xzxhh5ue+9pDnrLrDQCfx6ws2x/F7BosjYRcUDSK8CvAz8ubyRpGbAMYGBgIIXSzKqXyxf42ENb2DdWexeOJzuzdtDT6gLKRcTaiBiMiME5c+a0uhzrIrl8geXrnnLYW0dL4wq/AMwv2z+h9FqlNrskvQF4E8Wbt2YtV28XjoPe2k0agb8JWCjpRIrBfjlw5YQ264GlwPeAS4Bvuf/eWi2XL3D9Q1sYrfGqvr+vh1svdl+9tZ/EgV/qk78WGKI4LPOeiNgq6WZgOCLWA/8AfFHSduCnFH8pmLXMolse5cWf76/5fXOPnsHGlec2oCKzxktlHH5EPAI8MuG1G8q2fwm8L41zmSVR71z14LC39ucnba0rJBlXP6u/j5sWe/4ba38OfOto9d6QBThyRi+3/NmpDnrrGA5861j1hr2D3jqVA986lic7MzucA986Tr3z1TvsrdM58K0j5PIFlj+4mToelHXQW9dw4FtbSzL6xsMsrds48K1tJVlq0NMiWDdy4FtbWpUbqSvsf61X/PstFzSgIrPsc+BbW0kyhbH76q3bOfCtbdTbheO+erMiB75lnkfgmKXDgW+ZVgz7p2oO+4W/cSSPXnd2Q2oya1cOfMukVbkR7tu4g1pXTfAC4maTc+Bb5tQ7hbGHWppNzYFvmVLPwiS9gjsu9VW92XQSBb6kY4AHgAXA88ClEfFyhXbfBM4CvhMR/znJOa0z1TuzpW/KmlUv6RX+CuDxiFgtaUVp/6MV2q0BZgL/LeH5rAPVGvazZ/Zx44VekMSsVkkD/yLg7NL2vcATVAj8iHhc0tkTX7fulssXWDO0jcLe0ara9wjudNeNWd2SBv7ciHihtP0jYG7Cz7MuUM8DVO6nN0tu2sCX9Bjw5gqHVpbvRERIqnEQ3evOtQxYBjAwMJDkoyyDcvkCH1n3FPsP1vZj4idlzdIxbeBHxDmTHZP0oqTjIuIFSccBLyUpJiLWAmsBBgcHE/3ysGypd1oED7U0S0/SLp31wFJgdenr1xJXZB2n1rDvlbhi0XyPvjFLWdLAXw18VdI1wA+BSwEkDQIfjIgPlPb/Ffht4ChJu4BrImIo4bktw2q9ITvuM35K1qxhEgV+RPwEeE+F14eBD5TtvzPJeax95PIFblq/lb2jYzW/9x1vPcZhb9ZAftLWUlPvw1PgB6jMmsGBb6moJ+wFfNpdOGZN48C3xOqZ7KyvB9a8z2Fv1kwOfEvkqr//Xs1h76GWZq3hwLe61HNz1sMtzVrLgW81qSfofUPWLBsc+FaVep6UFXCVw94sMxz4NqVcvsBfPbC55vd5TVmz7HHg26Q8/41ZZ3Hg2+vk8gU++o9bePXAazW9r69XrLnkbR5qaZZRDnw7TC5fYPm6pxircQpjr0Jlln0OfAOKT8p+eeMOXqtxUmpPdmbWPhz4Vndf/dVnDTjszdpIT6sLsNbK5Qt1h72HW5q1F1/hd7k1Q9uqbuvRN2btzYHfZRbd8igv/nx/Te/xrJZmncFdOl2knrDv65XD3qxDJAp8ScdIelTSM6Wvsyu0OV3S9yRtlbRF0mVJzmm1y+ULvGP1t2oO+9kz+zyu3qyDJO3SWQE8HhGrJa0o7X90Qpt9wH+JiGckHQ88KWkoIvYmPLdVIZcvsPzBpxibZrzlvFn97N47yvGz+ll+3kkOebMOlDTwLwLOLm3fCzzBhMCPiB+Ube+W9BIwB3DgN1Cti4h/d8UfNbgiM2u1pIE/NyJeKG3/CJg7VWNJZwIzgGcnOb4MWAYwMDCQsLTulcsXuP6hEUbHDlbVfu7RMxpckZllwbSBL+kx4M0VDq0s34mIkDRpv4Gk44AvAksjouIkLRGxFlgLMDg4WOMzn5bLF/j4P23l5X3Vz1U/9+gZbFx5bgOrMrOsmDbwI+KcyY5JelHScRHxQinQX5qk3RuBrwMrI2JD3dVaRfWsKTt7Zh/5G/64QRWZWRYlHZa5Hlha2l4KfG1iA0kzgIeBL0TEuoTnswkW3fJo7QuI94obLzylQRWZWVYlDfzVwLmSngHOKe0jaVDS3aU2lwLvAt4vaXPpz+kJz9v1cvkCZ9z8L1UPtVTp67xZ/R5qadalFJHNrvLBwcEYHh5udRmZVOtN2V6JOy51yJt1A0lPRsRgpWN+0rYNrRnaVnXY9/U47M2syIHfhnZXObZ+Vn8fa97nsDezIk+e1oaOn9U/5QNVHmppZpU48DNsVW6Er2zcycEIeiWuWDSfTy45leXnnVSxD39Wfx83LfYyg2ZWmQM/g3L5Ah97aAv7xn71fNrBCL60YQfAoYVH1gxt8/w3ZlY1j9LJmOlG4PRKPHvrBU2uyszahUfptJHpRuAczOgvaDPLPgd+xkw3AqdXmvK4mdlk3IffQuNTGJf3w083AueKRfObWKGZdRJf4bfIqtwIf/3AZgp7RwmgsHeU6x8a4Q9/ew79fb2vay/B1WcNHLpha2ZWK1/hN9F0i5KMjh3k2/++h1svPtUjcMwsdQ78JlmVG+G+DTuY7pbr7r2jLDljngPezFLnLp0myOULVYU9FJ+iNTNrBAd+E6wZ2lZV2AtYft5JjS7HzLqUA78JqpnsTMBVZw24K8fMGsZ9+E0w3VDLeb4xa2ZN4MBvgkqTnY1f0XuYpZk1S6LAl3QM8ACwAHgeuDQiXp7Q5i0U17TtAfqA/xURn09y3nYzfuXuoZZm1kqJJk+TdDvw04hYLWkFMDsiPjqhzYzSeV6VdBTwfeA/RcTuqT67WydPMzNLopGTp10E3FvavhdYMrFBROyPiFdLu0ekcE4zM6tD0vCdGxEvlLZ/BMyt1EjSfElbgJ3AbZNd3UtaJmlY0vCePXsSlmZmZuWm7cOX9Bjw5gqHVpbvRERIqtg/FBE7gdMkHQ/kJK2LiBcrtFsLrIVil04V9ZuZWZWmDfyIOGeyY5JelHRcRLwg6TjgpWk+a7ek7wPvBNbVXK2ZmdUtaZfOemBpaXsp8LWJDSSdIKm/tD0b+ANgW8LzmplZjZKOw18NfFXSNcAPgUsBJA0CH4yIDwD/Ebij1N0j4G8jYiTheZui0nz1HkppZu0qUeBHxE+A91R4fRj4QGn7UeC0JOdphYlry47PVw849M2sLflJ25KJV/P79h943dqyo2MHWTO0zYFvZm3JgU/lq/nJVDMRmplZFvkhKIpTHky8mp+M56s3s3blwKf6q/b+vl7PV29mbcuBz+RX7bP6+5g3qx9RnML41otPdf+9mbUt9+FTefri/r5eblp8igPezDqGAx9PX2xm3aGjA7+WB6eWnDHPAW9mHa1jA98PTpmZHa5jb9pWGmo5/uCUmVk36tjAn2yopR+cMrNu1bGBP9lQSz84ZWbdqmMDf/l5J9Hf13vYa35wysy6WcfetPVQSzOzw3Vs4IOHWpqZlevYLh0zMztcosCXdIykRyU9U/o6e4q2b5S0S9LnkpzTzMzqk/QKfwXweEQsBB4v7U/mE8D/SXg+MzOrU9LAvwi4t7R9L7CkUiNJvwfMBf4l4fnMzKxOSQN/bkS8UNr+EcVQP4ykHuAO4MPTfZikZZKGJQ3v2bMnYWlmZlZu2lE6kh4D3lzh0MrynYgISVGh3YeARyJil6QpzxURa4G1AIODg5U+y8zM6jRt4EfEOZMdk/SipOMi4gVJxwEvVWj2duCdkj4EHAXMkPT/ImKq/n4zM0tZ0nH464GlwOrS169NbBARV41vS3o/MOiwNzNrvqR9+KuBcyU9A5xT2kfSoKS7kxZnZmbpUUQ2u8ol7QF+OOHlY4Eft6Ccarm+ZFxfMq4vmU6p7y0RMafSgcwGfiWShiNisNV1TMb1JeP6knF9yXRDfZ5awcysSzjwzcy6RLsF/tpWFzAN15eM60vG9SXT8fW1VR++mZnVr92u8M3MrE4OfDOzLpHpwM/6fPvV1CfpLZL+TdJmSVslfTBj9Z0u6Xul2rZIuixL9ZXafVPSXkn/3KS6zpe0TdJ2Sa97KlzSEZIeKB3fKGlBM+qqob53lX7mDki6pJm1VVnfdZKeLv28PS7pLRmr74OSRkr/Zr8j6eQs1VfW7r2SQlL1QzUjIrN/gNuBFaXtFcBtU7T9LPBl4HNZqg+YARxR2j4KeB44PkP1/QdgYWn7eOAFYFZW6isdew9wIfDPTaipF3gW+M3S391TwMkT2nwI+Hxp+3LggSb+zFVT3wLgNOALwCXNqq2G+v4QmFna/osMfv/eWLa9GPhmluortTua4voiGyhOV1PV52f6Cp/sz7c/bX0RsT8iXi3tHkFz/1dVTX0/iIhnStu7KU6AV/EpvVbUV6rrceDnTarpTGB7RDwXEfuB+ynWWa687nXAezTdVLBNrC8ino+ILcBrTaqp1vq+HRH7SrsbgBMyVt/PynaPBJo5sqWanz8oLih1G/DLWj4864Gf6nz7DTBtfQCS5kvaAuykeBW7O0v1jZN0JsWrimcbXVhJTfU1yTyKf0/jdpVeq9gmIg4ArwC/3pTqqquvlWqt7xrgGw2t6HBV1SfpLyU9S/F/of+9SbVBFfVJ+l1gfkR8vdYPTzpbZmLNnG+/RfURETuB0yQdD+QkrYuIF7NSX+lzjgO+CCyNiNSuDNOqzzqPpKuBQeDdra5looi4C7hL0pXAKoqzAbdc6QL3TuD99by/5YEfGZ9vP4X6yj9rt6TvA++k2BWQifokvRH4OrAyIjakUVea9TVZAZhftn9C6bVKbXZJegPwJuAnzSmvqvpaqar6JJ1D8Zf+u8u6PJuh1u/f/cD/bmhFh5uuvqOB3wGeKF3gvhlYL2lxRAxP9+FZ79IZn28fpphvPyIGImIBxW6dL6QV9mnUJ+kESf2l7dnAHwDbMlTfDOBhit+3VH4J1WDa+lpgE7BQ0oml783lFOssV173JcC3onQnLSP1tdK09Uk6A/g7YHFENPuXfDX1LSzb/VPgmazUFxGvRMSxEbGglHkbKH4fpw378Q/I7B+K/aKPU/yGPwYcU3p9ELi7Qvv309xROtPWB5wLbKF4t30LsCxj9V0NjAGby/6cnpX6Svv/CuwBRin2aZ7X4LouAH5A8V7GytJrN5f+YQH8GvAgsB34v8BvNuvvtMr6fr/0ffoFxf95bM1YfY8BL5b9vK3PWH2fBbaWavs2cEqW6pvQ9glqGKXjqRXMzLpE1rt0zMwsJQ58M7Mu4cA3M+sSDnwzsy7hwDcz6xIOfDOzLuHANzPrEv8f4FfHlv0lopIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x=np.reshape(attributions, -1),\n",
    "            y=np.reshape(np.sum(interactions, axis=-1), -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_explainer = PathExplainerTF(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:00<00:00, 47.12it/s]\n"
     ]
    }
   ],
   "source": [
    "orig_attributions = orig_explainer.attributions(inputs=data,\n",
    "                                      baseline=baseline,\n",
    "                                      batch_size=50,\n",
    "                                      num_samples=200,\n",
    "                                      use_expectation=False,\n",
    "                                      output_indices=0,\n",
    "                                      verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [01:48<00:00, 10.80s/it]\n"
     ]
    }
   ],
   "source": [
    "orig_interactions = orig_explainer.interactions(inputs=data,\n",
    "                                      baseline=baseline,\n",
    "                                      batch_size=50,\n",
    "                                      num_samples=200,\n",
    "                                      use_expectation=False,\n",
    "                                      output_indices=0,\n",
    "                                      verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "orig_attributions_summed = np.sum(orig_attributions, axis=2)\n",
    "orig_interactions_summed = np.sum(orig_interactions, axis=(2, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f7075c212d0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAa9klEQVR4nO3dfZRc9X3f8fdnlxVZniwRMA9Ca2FXJYWAIdkAKbFNaskocY2ozTO0cIqr4zicNgUriCOVJ8exsBqCT82po1LOwY4THhQYrwNGAWHa2MeiWjJ6sGhkCQ5GGmGQbYRTWMNK+vaPmZVHy8zuzNw7M3dmPq9zdPbeub+Z+z2r1Ud3f/d3fz9FBGZm1v362l2AmZm1hgPfzKxHOPDNzHqEA9/MrEc48M3MesQh7S6gmmOOOSbmzp3b7jLMzDrKc88995OIOLbSscwG/ty5cxkdHW13GWZmHUXSj6odc5eOmVmPcOCbmfUIB76ZWY9IJfAlLZS0VdJ2SUunaPcpSSFpOI3zmplZ7RIHvqR+4B7g94BTgSsknVqh3ZHAfwKeTXpOMzOrXxpX+GcD2yPixYh4B3gAWFSh3eeBO4FfpHBOMzOrUxqBPxvYUba/s/TaAZJ+A5gTEY9N9UGSFksalTS6e/fuFEozM7MJTb9pK6kPuAu4cbq2EbEqIoYjYvjYYys+N2BmZg1KI/ALwJyy/ZNKr004Evh14BlJLwHnAiO+cWtm1lppPGm7Hpgn6WSKQX85cOXEwYh4AzhmYl/SM8DnIsKP0ZpZT8vlC6xcs5Vde8Y4ceYgSy44hYvOmj39GxuUOPAjYq+k64E1QD9wX0RskXQHMBoRI0nPYWbWbXL5Ajc/spmx8X0AFPaMcfMjmwGaFvqpzKUTEY8Dj0967ZYqbc9P45xmZp1s5ZqtB8J+wtj4Plau2ZrtwDczs+mVd+FUW018156xpp3fgW9m1gLLc5v5xrqXqwb9hBNnDjatBs+lY2bWZLl8oaawHxzoZ8kFpzStDl/hm5k12co1W6cMe0FnjNIxM7OpTdUvP3vmIN9b+q9aUoe7dMzMmqxav7ygqV04kznwzcyabMkFpzA40H/QawKuOneoqV04k7lLx8ysySZCvZVP1VbiwDcza4GLzprd8oCfzF06ZmY9woFvZtYjHPhmZj3CffhmZlNYntvMXz+7g30R9Etccc4c/uSi09tdVkMc+GZmVSzPbeYv1718YH9fxIH9Tgx9B76ZWZlaZrT862d3OPDNzDrZ5EVJqtkX002Dlk2+aWtmVlJpUZJK+qUWVJO+VAJf0kJJWyVtl7S0wvHPSNosaYOk70o6NY3zmpmlqdbFR644Z06TK2mOxF06kvqBe4AFwE5gvaSRiHi+rNlfRcRXS+0vBO4CFiY9t5lZEpMXEX/P4AB7xsartvcoHTgb2B4RLwJIegBYBBwI/Ij4eVn7w2HadQDMzJqq0iLiA/1ioE+M7/9lRA0O9PPFT57e9mkR0pBG4M8GdpTt7wTOmdxI0h8CNwAzgIqTP0taDCwGGBoaSqE0M7PKKvXXj+8LZh02wGEzDmnrJGfN0rJROhFxD3CPpCuB5cA1FdqsAlYBDA8P+7cAM2uaav31e94aJ3/Lx1pcTWukcdO2AJTfwTip9Fo1DwAXpXBeM7OGVVuUpJmLiLdbGoG/Hpgn6WRJM4DLgZHyBpLmle1+HNiWwnnNzBpWaVGSZi8i3m6Ju3QiYq+k64E1QD9wX0RskXQHMBoRI8D1kuYD48DrVOjOMTNrpawsStJKiow+MTY8PByjo6PtLsPMOsjkYZbdHuCVSHouIoYrHfPUCmbWFSoNs7z5kc0APRf61XhqBTPrCpWGWY6N72Plmq1tqih7fIVvZh1pcvdNocowy1qnS+gFDnwz6ziVum9E5Uf4u3mYZb0c+GbWUXL5Ajc+tPFdUxQHvCv0u32YZb3ch29mHWPiyr7afPQBzJ45iEpfu2UOnLT4Ct/MOsZ089XPnjnI95ZWnKrL8BW+mXWQqW7Auvtmeg58M+sY1W7A9kvuvqmBu3TMLDNy+QK3f2sLr79VXIRk5uAAt1142oEgX3LBKe9ac7ab5qtvNge+mWXCVf/j+3zvhZ8d9NqesXGWPLwRKD4t24vz36TJgW9mbVcp7CeM7w9Wrtl6INTLg9/q48A3s7bJ5Qvc8OAG9k/Tzk/LpsOBb2ZtkcsX+KMHN9TU1k/LpsOBb2YtU3xwahNj49Nd0x/Mwy3T4cA3s5ZYntvMX657ue73XX3ukPvsU5JK4EtaCHyZ4opX90bEiknHbwA+DewFdgP/PiJ+lMa5zSzbJg+1rNVAH6y85EyHfYoSB76kfuAeYAGwE1gvaSQini9rlgeGI+ItSX8AfAm4LOm5zSzbluc28411L1ecxXIq8957OE/ecH4zSuppaVzhnw1sj4gXASQ9ACwCDgR+RHynrP064OoUzmtmGZXLF1j26GbefKf6vDfVnPeBo/nGf/jtJlRlaQT+bGBH2f5O4Jwp2l8HfLvSAUmLgcUAQ0NDKZRmZq021Zj6aq4+d4g/uej0JlVkE1p601bS1cAw8JFKxyNiFbAKiouYt7A0M0vBgrueYdtrb9bcXsBVDvuWSSPwC8Ccsv2TSq8dRNJ8YBnwkYh4O4XzmlmGLM9trjnsJ8+RY62RRuCvB+ZJOpli0F8OXFneQNJZwF8ACyPitRTOaWYZUL6ubK2/krv7pn0SB35E7JV0PbCG4rDM+yJii6Q7gNGIGAFWAkcAD0sCeDkiLkx6bjNrj1y+wG0jW9gzVt9QS4d9e6XShx8RjwOPT3rtlrLt+Wmcx8zab/IC4rU67wNHO+zbzE/amlnNGhmBA76yzwoHvplNq5Gg75e44pw5DvoMceCbWVW5fIEbH9rAvjoGSc8cHGDDrR9rXlHWMK9pa2YV5fIFlqzeWFfYD/SJ2y48rXlFWSK+wjezg+TyBf549UbeqSfpgdlebjDzHPhmdkAjUxh77pvO4cA3M6CxG7Pz3nu4w76DOPDNjOW5zXWFfR9w12Weq77TOPDNelS9E52B58DpdA58sx7USNi7r77zOfDNekijyw067LuDA9+sR3gEjjnwzXpAvWHvaRG6kwPfrEtNzFVf2DNW1/vu9uibruXAN+syuXyBJQ9vYHx/fe/zUMvu58A36yK5fIEbHtxAPVk/0CdWXvJBB30PcOCbdYlGbsoODvTxxU+e4bDvEanMlilpoaStkrZLWlrh+Icl/YOkvZIuTuOcZlaUyxc49b98u66wF8VFSf7v53/PYd9DEl/hS+oH7gEWADuB9ZJGIuL5smYvA9cCn0t6PjP7pUYeoPKslr0rjS6ds4HtEfEigKQHgEXAgcCPiJdKx+q8jWRmlTQyhbHH1FsagT8b2FG2vxM4p5EPkrQYWAwwNDSUvDKzLpPLF7htZAt7xup7UtZryhpk7KZtRKwCVgEMDw/Xt/qCWZdrZPriPsFdl3qopRWlEfgFYE7Z/kml18wsBbl8gc89vJG9++u7BjpsoI8/9QgcK5NG4K8H5kk6mWLQXw5cmcLnmvW8XL7AjQ9vZF8dYT/vvYfz5A3nN68o61iJAz8i9kq6HlgD9AP3RcQWSXcAoxExIum3gEeBWcAnJN0eEV7p2KyKXL7ATX+zibf31jfOwTdmbSqp9OFHxOPA45Neu6Vsez3Frh4zm0IuX+CGhzZQT+/NrMMGuPUTXpTEppepm7ZmvSyXL/BHD26o6z2+ord6OPDNMsALiFsrOPDN2qiRoBdwlcfVWwMc+GZt4nVlrdUc+GYt1Ohc9eAuHEvOgW/WIo1030CxC+fPvTCJpcCBb9YCjYa958CxNDnwzZqokUVJwH311hwOfLMmyOUL3PjQBuqYvRjwtAjWXA58s5Q1MvoG3H1jzefAN0tJo903vilrreLAN0vBOV94klf/6Z263+e+emslB75ZAo2Ovhnog5WX+KreWsuBb9agM259gp+/va/u97mv3trFgW9Wp0ZvynoEjrWbA9+sRo1MXzzBV/WWBQ58sxp4AXHrBqkEvqSFwJcpLnF4b0SsmHT8UOBrwG8CPwUui4iX0ji3WTM1elP2qEP72XT7wiZUZNa4xIEvqR+4B1gA7ATWSxqJiOfLml0HvB4R/0zS5cCdwGVJz23WTHOXPtbQ+9xXb1mVxhX+2cD2iHgRQNIDwCKgPPAXAbeVtlcDX5GkiKjzwXOz5mv0AapDD+njzk+d4S4cy6w0An82sKNsfydwTrU2EbFX0hvArwI/KW8kaTGwGGBoaCiF0szq41ktrZv1tbuAchGxKiKGI2L42GOPbXc51mMc9tbt0rjCLwBzyvZPKr1Wqc1OSYcA76F489as7RrtwnHQW6dJI/DXA/MknUwx2C8HrpzUZgS4Bvg+cDHwtPvvrd0c9NZrEgd+qU/+emANxWGZ90XEFkl3AKMRMQL8T+DrkrYDP6P4n4JZW/gBKutVqYzDj4jHgccnvXZL2fYvgEvSOJdZEo3OaunJzqwb+Elb6wmNXtV7+mLrJg5863qeq96syIFvXavRm7KHz+jnC//mdHffWNdx4FvXyeUL3PzIJsbG99f93ru91KB1MQe+dZVcvsAND21gfwODfh321u0c+NYVcvkCSx7eQAMX9Q566xkOfOt4jS41eNyRM3h22YImVGSWTQ5861iNzn0DHoFjvcmBbx2p0bD/lX7xj1/4/SZUZJZ9DnzrKJ7/xqxxDnzrGAvueoZtr71Z9/vmvfdwh70ZDnzrAElG4PjK3uyXHPiWacWw31h32PumrNm7OfAtk3L5Ard/awuvvzVe1/u8gLhZdQ58yxwvNWjWHA58y5RGZrac0S++dPEH/bSs2TQSBb6ko4EHgbnAS8ClEfF6hXZPAOcC342If53knNadGhlu6UVJzOrTl/D9S4G1ETEPWFvar2Ql8G8Tnsu6VL1hP+uwAe6+7Ey2/enHHfZmdUjapbMIOL+0fT/wDHDT5EYRsVbS+ZNft96WyxdYuWYrhT1jNb/HE52ZNS5p4B8XEa+Utn8MHJfkwyQtBhYDDA0NJSzNsirJTVmHvVnjpg18SU8Bx1c4tKx8JyJCUgOzkB/0GauAVQDDw8OJPsuyp9FpEQT8ua/szRKbNvAjYn61Y5JelXRCRLwi6QTgtVSrs67R6LQIfoDKLD1Ju3RGgGuAFaWv30xckXWVRqZF6Je44pw5HlNvlrKkgb8CeEjSdcCPgEsBJA0Dn4mIT5f2/x74NeAISTuB6yJiTcJzW4Y1+qSsb8qaNU+iwI+InwIfrfD6KPDpsv0PJTmPdZbiIuKbGRuvbxWq8z5wtMPerIn8pK2lJpcv8MerN/LOvvrvt3taBLPmc+BbKhoZgeOJzsxay4FviTUyAsejb8xaz4FvDcvlC9z0N5t4e299k9W7+8asPRz41pDluc18Y93L1NNbP3vmIEsuOMU3Zs3axIFvdcnlC9w2soU9Y7UPt3T3jVk2OPCtJo3Of+OwN8sOB75N64xbn+Dnb9c3pt4jcMyyJ+l8+NbFcvkC71/6WN1hf94HjnbYm2WQr/DtXXL5Asse3cyb79Qe9AKu8ugbs0xz4NtBGnmAatZhA9z6idM8+sYs4xz4BjR2VQ+e7MyskzjwreGFSbwClVln8U3bHpfLFxoOe/fXm3UWX+H3uJVrttbV3kFv1rkc+D2m0aUGfWPWrPM58HtII2HvK3qz7pGoD1/S0ZKelLSt9HVWhTZnSvq+pC2SNkm6LMk5rX65fIHzVjztsDfrcUlv2i4F1kbEPGBtaX+yt4B/FxGnAQuBuyXNTHheq1EuX2DJ6o0U9oxN2W7m4MCB7VmHDXD3ZWc67M26TNIunUXA+aXt+4FngJvKG0TED8u2d0l6DTgW2JPw3DaFXL7AyjVbpw36CRtu/ViTKzKzdksa+MdFxCul7R8Dx03VWNLZwAzghSrHFwOLAYaGhhKW1rvqXUTcY3PNesO0gS/pKeD4CoeWle9EREiquh6GpBOArwPXRETFJZIiYhWwCmB4eLj+lbB7XC5f4PZvbeH1t2qfqx7grsvObFJFZpYl0wZ+RMyvdkzSq5JOiIhXSoH+WpV2RwGPAcsiYl3D1VpFjaw+BV6ByqzXJO3SGQGuAVaUvn5zcgNJM4BHga9FxOqE57NJGlmYZKBPrLzkgw56sx6TtPt2BbBA0jZgfmkfScOS7i21uRT4MHCtpA2lP+5DSCiXL3DWHX9Xc9ir9HX2zEGHvVmPUkQ2u8qHh4djdHS03WVkUr03Zfsl/uxSh7xZL5D0XEQMVzrmARodaOWarTWH/UCfw97Mihz4HWhXjWPrZw4OuPvGzA7wXDoZNvHw1K49Y5xYNqLmxJmD0z5Q5WkRzGwyX+FnVPmUCAEU9oyxZPXG4usXnMLgQH/F9/XJYW9mlfkKP4Ny+QI3PLSB/ZPup4/vC27/1hbytxSnQah09W9mVo0DP2MmRuBMDvsJE0/RXnTWbAe8mdXFXToZU88IHDOzejjwM2a6ETjl0xibmdXDXTptVGkUzlQjcAb6xG0XntbiKs2sW/gKv02W5zbznx/ccNAonJsf2czv/tqxFUfgeEy9mSXlK/wWmm5RkrHxfXznH3fzxU+e7hE4ZpY6B36L1DqF8a49Yx6BY2ZN4S6dFsjlCzXPV3/izMGm12NmvcmB3wIr12ytKewFLLnglGaXY2Y9yoHfArVMdibgqnOH3JVjZk3jPvwWmG6yMy81aGat4MBvgSUXnPKuBUsmrug9yZmZtUqiwJd0NPAgMBd4Cbg0Il6f1OZ9FNe07QMGgP8WEV9Nct5OM3Hl7qGWZtZOiZY4lPQl4GcRsULSUmBWRNw0qc2M0nnelnQE8APgX0bErqk+20scmpnVr5lLHC4C7i9t3w9cNLlBRLwTEW+Xdg9N4ZxmZtaApOF7XES8Utr+MXBcpUaS5kjaBOwA7qx2dS9psaRRSaO7d+9OWJqZmZWbtg9f0lPA8RUOLSvfiYiQVLF/KCJ2AGdIOhHISVodEa9WaLcKWAXFLp0a6jczsxpNG/gRMb/aMUmvSjohIl6RdALw2jSftUvSD4APAavrrtbMzBqWtEtnBLimtH0N8M3JDSSdJGmwtD0L+B1ga8LztkQuX+C8FU9z8tLHOG/F0+TyhXaXZGbWsKSBvwJYIGkbML+0j6RhSfeW2vwL4FlJG4H/BfzXiNic8LxNN7HU4OTpix36ZtapEo3Dj4ifAh+t8Poo8OnS9pPAGUnO0w6VlhocG9/HyjVbPX7ezDqSh0hWUW3+m1rmxTEzyyJPrVAyebnBmYcN8Ppb4+9q5+mLzaxTOfD5ZX/9RBdOYc8YA31ioF+M7/vl6NDBgX5PX2xmHctdOlTurx/fHxw+4xBmzxxEFGe0/OInT3f/vZl1LF/hU71f/o2xcTbc+rEWV2Nm1hy+wqd6v7z7682smzjwKc5XPzjQf9Br7q83s27T1V06k0feVJuD3vPVm1kv6NrArzTy5uZHig/4Vgt9B7yZdbOu7dKZ6klZM7Ne1LWB7ydlzcwO1rWB75E3ZmYH69rA98gbM7ODde1NW4+8MTM7WNcGPnjkjZlZua7t0jEzs4MlCnxJR0t6UtK20tdZU7Q9StJOSV9Jck4zM2tM0iv8pcDaiJgHrC3tV/N54H8nPJ+ZmTUoaeAvAu4vbd8PXFSpkaTfBI4D/i7h+czMrEFJA/+4iHiltP1jiqF+EEl9wJ8Bn5vuwyQtljQqaXT37t0JSzMzs3LTjtKR9BRwfIVDy8p3IiIkRYV2nwUej4idkqY8V0SsAlYBDA8PV/osMzNr0LSBHxHzqx2T9KqkEyLiFUknAK9VaPbbwIckfRY4Apgh6f9FxFT9/WZmlrKk4/BHgGuAFaWv35zcICKumtiWdC0w7LA3M2u9pH34K4AFkrYB80v7SBqWdG/S4szMLD2KyGZXuaTdwI8mvXwM8JM2lFMr15eM60vG9SXTLfW9LyKOrXQgs4FfiaTRiBhudx3VuL5kXF8yri+ZXqjPUyuYmfUIB76ZWY/otMBf1e4CpuH6knF9ybi+ZLq+vo7qwzczs8Z12hW+mZk1yIFvZtYjMh34WZ9vv5b6JL1P0j9I2iBpi6TPZKy+MyV9v1TbJkmXZam+UrsnJO2R9LctqmuhpK2Stkt611Phkg6V9GDp+LOS5rairjrq+3DpZ26vpItbWVuN9d0g6fnSz9taSe/LWH2fkbS59G/2u5JOzVJ9Ze0+JSkk1T5UMyIy+wf4ErC0tL0UuHOKtl8G/gr4SpbqA2YAh5a2jwBeAk7MUH3/HJhX2j4ReAWYmZX6Ssc+CnwC+NsW1NQPvAC8v/R3txE4dVKbzwJfLW1fDjzYwp+5WuqbC5wBfA24uFW11VHf7wKHlbb/IIPfv6PKti8EnshSfaV2R1JcX2Qdxelqavr8TF/hk/359qetLyLeiYi3S7uH0trfqmqp74cRsa20vYviBHgVn9JrR32lutYC/9Sims4GtkfEixHxDvAAxTrLlde9GvioppsKtoX1RcRLEbEJ2N+imuqt7zsR8VZpdx1wUsbq+3nZ7uFAK0e21PLzB8UFpe4EflHPh2c98FOdb78Jpq0PQNIcSZuAHRSvYndlqb4Jks6meFXxQrMLK6mrvhaZTfHvacLO0msV20TEXuAN4FdbUl1t9bVTvfVdB3y7qRUdrKb6JP2hpBco/hb6H1tUG9RQn6TfAOZExGP1fnjS2TITa+V8+22qj4jYAZwh6UQgJ2l1RLyalfpKn3MC8HXgmohI7cowrfqs+0i6GhgGPtLuWiaLiHuAeyRdCSynOBtw25UucO8Crm3k/W0P/Mj4fPsp1Ff+Wbsk/QD4EMWugEzUJ+ko4DFgWUSsS6OuNOtrsQIwp2z/pNJrldrslHQI8B7gp60pr6b62qmm+iTNp/if/kfKujxbod7v3wPAf29qRQebrr4jgV8Hnild4B4PjEi6MCJGp/vwrHfpTMy3D1PMtx8RQxExl2K3ztfSCvs06pN0kqTB0vYs4HeArRmqbwbwKMXvWyr/CdVh2vraYD0wT9LJpe/N5RTrLFde98XA01G6k5aR+tpp2voknQX8BXBhRLT6P/la6ptXtvtxYFtW6ouINyLimIiYW8q8dRS/j9OG/cQHZPYPxX7RtRS/4U8BR5deHwburdD+Wlo7Smfa+oAFwCaKd9s3AYszVt/VwDiwoezPmVmpr7T/98BuYIxin+YFTa7r94EfUryXsaz02h2lf1gAvwI8DGwH/g/w/lb9ndZY32+Vvk9vUvzNY0vG6nsKeLXs520kY/V9GdhSqu07wGlZqm9S22eoY5SOp1YwM+sRWe/SMTOzlDjwzcx6hAPfzKxHOPDNzHqEA9/MrEc48M3MeoQD38ysR/x/pS3KZIq1D34AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x=np.reshape(attributions, -1),\n",
    "            y=np.reshape(orig_attributions_summed, -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f7075c21650>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAa2klEQVR4nO3df7BU5Z3n8feH68W5Gs2ViESQGxzDOmtGR2d6xJSTxE1gdZINOMb4I1iLtTqsk7F2ZsmwwYLy16wLhh2zqYo1G+KmymRN/LWmQ0YmLJK4O0kJyzUXvcEpAroqNEQZI4kbbxTwu390X9Jc+nK7+5zuPt39eVVRnNP9dJ+vgB8envOc51FEYGZmnW9SqwswM7PmcOCbmXUJB76ZWZdw4JuZdQkHvplZlziu1QWM59RTT41Zs2a1ugwzs7by9NNP/1NETK30XmYDf9asWQwODra6DDOztiLppfHe85COmVmXcOCbmXUJB76ZWZdw4JuZdQkHvplZl3Dgm5l1CQe+mVmXSCXwJV0mabuknZKWHaPdpySFpFwa1zUzs+olDnxJPcC9wB8D5wDXSjqnQruTgL8ANie9ppmZ1S6NJ20vBHZGxAsAkh4EFgDPjWn318DdwNIUrmlm1hHyQwVWr9/Onv0jTO/vY+mlZ3P5BTMacq00hnRmALvKzneXXjtM0u8DMyPi8WN9kaTFkgYlDe7bty+F0szMsis/VOCWx4Yp7B8hgML+EW55bJj8UKEh12v4TVtJk4B7gM9N1DYi1kRELiJyU6dWXPvHzKxjrF6/nZEDh454beTAIVav396Q66UR+AVgZtn5GaXXRp0E/C7wpKQXgYuAtb5xa2bdbs/+kZpeTyqNwN8CzJZ0pqTJwDXA2tE3I+IXEXFqRMyKiFnAJmB+RHgpTDPratP7+2p6PanEgR8RB4GbgfXAPwIPR8Q2SXdKmp/0+83MOtXSS8+mr7fniNf6entYeunZDbleKuvhR8Q6YN2Y124dp+0laVzTzKzdjc7GadYsncxugGJm1g0uv2BGwwJ+LC+tYGbWJRz4ZmZdwoFvZtYlHPhmZl3CgW9m1iU8S8fMrErNXOisERz4ZmZVWJEf5oFNLxOl89GFzoC2CX0P6ZiZTSA/VDgi7Ec1cqGzRnDgm5lNYPX67UeF/ahGLXTWCA58M7MJHCvUG7XQWSN4DN/MrGS8m7LT+/soVAh9QcMWOmsEB76Zdb38UIE7vruN1988cPi18puySy89m1seGz5isxIBCy8aaJsbtuDAN7MuN7rN4Nidp+A3N2V/tOyjQPNWtWwUB76ZdbVK2wyWGx2/b+aqlo3im7Zm1tUmmmXTTjdlJ5JK4Eu6TNJ2STslLavw/k2ShiVtlfRDSeekcV0zs6SOFeiN3H2qFRIHvqQe4F7gj4FzgGsrBPo3I+LciDgf+AJwT9LrmpmlodI2gwD9fb2svOLcth/GKZfGGP6FwM6IeAFA0oPAAuC50QYR8cuy9ifCuM8wmJk1VbO3GWylNAJ/BrCr7Hw3MGdsI0l/DiwBJgMfrfRFkhYDiwEGBgZSKM3MulUtC511wg3ZajTtpm1E3BsRZwGfB1aM02ZNROQiIjd16tRmlWZmHWZ0qmVh/wjBb+bU54cKrS6tpdII/AIws+z8jNJr43kQuDyF65qZHSU/VOBzDz9z1FTLdlvorBHSCPwtwGxJZ0qaDFwDrC1vIGl22ekngB0pXNfM7AijPftDUfk2YTstdNYIicfwI+KgpJuB9UAP8LWI2CbpTmAwItYCN0uaCxwAXgcWJb2umdlYEz1E1Ulz6uuRypO2EbEOWDfmtVvLjv8ijeuYmR3LsXrwnTanvh5eWsHM2k6tq1r2SB03p74eDnwzaysLv/oUP3r+54fPJ1rVsq+3x2Ff4rV0zKxtrMgPHxH2o0Zn4Fx+wQxWXnEuM/r7EDCjv89hX8Y9fDNrG9/avGvc9zppVctGceCbWWatyA/zrc27OBRBjzTudEvwDJxqOPDNLJNW5If575tePnx+rLCH9tpqsFUc+GaWGeWzb2pZYfHis6Z4GKcKDnwzy4RjbTU41ujwTo/EtXNm8h8vP7cJFbY/B76Ztdzo+jcTDdtAMeyfX/nxJlTVeRz4ZtYy+aECt6/dxv6RA1V/5to5MyduZBU58M2sJWoZwgE8fJMCB76ZtcREC52Bn5JNmwPfzJoiP1Tgju9u4/U3qxu+8fo36XPgm1nD5YcKfO6RZzj0TnWTLd2zbwwHvpk1zNiFzqpxygm93PbJDzjsGyCVwJd0GfAlihug3BcRq8a8vwS4ETgI7AP+TUS8lMa1zSyb5ty1gVfeeLuqtoIJNxq35BIHvqQe4F5gHrAb2CJpbUQ8V9ZsCMhFxJuS/gz4AnB10mubWfbkhwp8/n88y1sH36mq/Yz+Pn607KMNrsogneWRLwR2RsQLEfE2xU3KF5Q3iIgfRMSbpdNNFDc6N7MOkx8qsPTRZ6oOe/AaOM2UxpDODKB8zdLdwJxjtL8B+PsUrmtmGVDPw1OjrrtowEM4TdTUm7aSrgNywEfGeX8xsBhgYGCgiZWZWT3GrmhZrRN6J/GfrjjPYd9kaQR+ASh/1vmM0mtHkDQXWA58JCLeqvRFEbEGWAOQy+VqWSzPzJqo+JTss4wcqH7oZtR1Fw34adkWSSPwtwCzJZ1JMeivAT5T3kDSBcBXgMsi4tUUrmlmLVJvr/7is6bwwJ9+sAEVWbUSB35EHJR0M7Ce4rTMr0XENkl3AoMRsRZYDbwLeEQSwMsRMT/ptc2seert1ff39XL7fM+rz4JUxvAjYh2wbsxrt5Ydz03jOmbWGvmhAkse3kqVD8rS2yNWX/l7DvmM8ZO2Zjauep6U9Q3Z7HLgm1lF8+55kh2v/qqmz/iGbLY58M3sKCvywzWF/YmTe7jrT7zYWdY58M3ssHoeonKvvn048M2s7qmWDvv24sA362K1LnQ2aoZXtmxLDnyzLlXPTdnZp53IhiWXNKYgazgHvlmXqWeqpYCFHr5pew58sy5ST6/eO1B1Dge+WZfIDxU8r77LOfDNOlx+qMDq9dsp7B+p+jPHHzeJuz/lp2U7jQPfrEPlhwrc8d1tvP5m9XPqJwH3XH2+g75DOfDNOkx+qMDSR7ZS61L1006azObl8xpTlGWCA9+sg9T7AJXDvjs48M06RD3TLcEbk3QTB75ZB6gl7Pt6e1h5hRc660aT0vgSSZdJ2i5pp6RlFd7/sKQfSzoo6co0rmnW7fJDBS5e9X1mLXu86rAXOOy7WOIevqQe4F5gHrAb2CJpbUQ8V9bsZeB64K+SXs/M6h+++aJn4HS1NIZ0LgR2RsQLAJIeBBYAhwM/Il4svVf7Fvdmdlh+qMDnHt7KoSq3GhzlXagM0gn8GcCusvPdwJx6vkjSYmAxwMDAQPLKzDpIPTNwfEPWyqUyhp+WiFgTEbmIyE2dOrXV5ZhlhsPe0pBGD78AzCw7P6P0mpklUM/uU+C16m18aQT+FmC2pDMpBv01wGdS+F6zruUevTVC4iGdiDgI3AysB/4ReDgitkm6U9J8AEl/KGk38GngK5K2Jb2uWada+NWnag772aed6LC3CaXy4FVErAPWjXnt1rLjLRSHesxsHPWsgeONSawWftLWrMXyQwWWf3uYX719qOrPOOitHg58sxaqZ6y+v6+X2+d7ByqrnQPfrAW80Jm1ggPfrMnef8vjHKzxSVnwdoOWnAPfrEnyQwX+8qGtNX/OvXpLiwPfrAnqGas/cXIPd/2JV7a09DjwzRqonqdl+3onsdILnVkDOPDNGqDem7Iep7dGcuCbpey8277HL9+qfk79qP/iteqtwRz4Zin5neXr+HWtC9VTfIjq/676RPoFmY2RqeWRzdrVrGWP1xX2006a7LC3pnEP3yyBOXdt4JU33q75c6ec0Mttn/TTstZcDnyzOtQ7fDP7tBPZsOSS9Asyq4ID36xGs5Y9XtfnHPbWag58syrVG/TgGTiWDQ58syrUG/aegWNZkkrgS7oM+BLQA9wXEavGvH888HXgD4DXgKsj4sU0rm3WSEl69X6IyrImceBL6gHuBeYBu4EtktZGxHNlzW4AXo+I90u6BrgbuDrptc0aqd6wP06wc6V79ZY9afTwLwR2RsQLAJIeBBYA5YG/ALi9dPwo8GVJiog6Fok1a6wkvfoXPXxjGZbGg1czgF1l57tLr1VsU9r0/BfAe8Z+kaTFkgYlDe7bty+F0syqtyI/7LC3jpapm7YRsQZYA5DL5dz7t6ZJEvTgsLf2kEbgF4CZZednlF6r1Ga3pOOAd1O8eWvWcu7VW7dIY0hnCzBb0pmSJgPXAGvHtFkLLCodXwl83+P31moewrFuk7iHHxEHJd0MrKc4LfNrEbFN0p3AYESsBf4b8A1JO4GfU/xLwaxlHPTWjVIZw4+IdcC6Ma/dWnb8a+DTaVzLLIl616oHz6u39pepm7ZmjVLPnrLl3Ku3TuDAt443754n2fHqr+r6rIPeOokD3zpWkqA/+fgenr3jspQrMmstB751nCTDN14WwTqZA986SpKw9/CNdTrvaWsdIz9UqCvsLz5risPeuoJ7+Nb2VuSHeWDTy9T6JN8k4B5vTGJdxIFvbSs/VODfP7S15qAH70Bl3cmBb22p3hk4F581hQf+9IMNqMgs+xz41lbqDXpPszTzTVtrI/WG/W/1yGFvhnv41gZW5If51uZdHKpxgVUJFs7x+jdmoxz4lmlz7trAK2+8XfPnfFPW7GgOfMukeh+gEvBFh71ZRQ58y5yFX32KHz3/85o/5+WLzY7NgW+ZUk/Yzz7tRDYsuaQxBZl1kESBL2kK8BAwC3gRuCoiXq/Q7nvARcAPI+JfJbmmdZ56n5Ttkbh2zkz36s2qlLSHvwzYGBGrJC0rnX++QrvVwAnAv014Pesw9U619E1Zs9olnYe/ALi/dHw/cHmlRhGxEXgj4bWsg+SHCpy57PGaw/64SXLYm9UpaQ9/WkTsLR3/DJiW5MskLQYWAwwMDCQszbKqnhk4xx83ibs/dZ6D3iyBCQNf0hPAeyu8tbz8JCJCUj3rWJV/xxpgDUAul0v0XZZN9dyU9fo3ZumYMPAjYu5470l6RdLpEbFX0unAq6lWZx2jnl69gIWeammWmqRDOmuBRcCq0s/fSVyRdZR659S7V2+WvqSBvwp4WNINwEvAVQCScsBNEXFj6fwfgN8B3iVpN3BDRKxPeG3LuHqWRfCTsmaNkyjwI+I14GMVXh8Ebiw7/1CS61j7WfjVp2oOe/fqzRrLT9paqvJDBe747jZef/NA1Z9x0Js1hwPfUlPrE7O+KWvWXA58S2RFfphvbn6Zd2qcROtevVnzOfCtbvUuYexVLc1aw4FvNcsPFfgPjz7D24dq69ZP7hFfuPL3PAPHrEUc+Fa1/FCBv3xoa82f6+/r5fb5H3DQm7WYA9+qUk/Y+6asWbY48K0qS2oM+1NO6OW2T7pXb5YlDnyryjtVtuvrncTKK7yqpVkWOfBtXCvyw3xr8y4ORXU3Zz3V0izbHPh2hHoXO5t92okOe7OMc+DbYd5u0KyzOfCN/FCB29duY/9I9evfQLFXv2HJJY0pysxS58DvYvmhAkse3lrTsgg9Es+v/HjjijKzhnHgd6l6l0W4ds7MBlRjZs3gwO9C+aFCzWHfI3HtnJl+iMqsjSUKfElTgIeAWcCLwFUR8fqYNucDfwucDBwC7oqIh5Jc1+qTHyqwev12CvtHqv6Mx+nNOkfSHv4yYGNErJK0rHT++TFt3gT+dUTskDQdeFrS+ojYn/DaVqV6NiXxk7JmnSdp4C8ALikd3w88yZjAj4iflh3vkfQqMBVw4DdBfqjALY8NM3LgUNWfmXbSZDYvn9fAqsysFZIG/rSI2Fs6/hkw7ViNJV0ITAaeH+f9xcBigIGBgYSldbd6p1o67M0614SBL+kJ4L0V3lpefhIRIWncCX6STge+ASyKiIpLs0TEGmANQC6Xq3EPJRuVHyqw9JFnOFDDfEsP4Zh1vgkDPyLmjveepFcknR4Re0uB/uo47U4GHgeWR8Smuqu1qqxev72qsO/r7WHlFec65M26xKSEn18LLCodLwK+M7aBpMnAt4GvR8SjCa9nVdhTxSyc/r5eh71Zl0k6hr8KeFjSDcBLwFUAknLATRFxY+m1DwPvkXR96XPXR0TtWydZVab394079bJH4m+u8jaDZt1IUeXSt82Wy+VicHCw1WW0pfHG8Ht7xGrvKWvW0SQ9HRG5Su/5Sds2U/7wVI/EoQhm9Pex9NKzDwf56M/ls3R8U9bMHPhtZOyc+tGNSQr7R7jlsWGAI0Lf4W5m5Rz4baCaJRFGDhxi9frtDnkzG5cDP+PyQwWWPvoMBw5NfK+lmtk5Zta9HPgZVut69dP7+xpbkJm1taTz8K1BRsfrqw37vt4ell56dmOLMrO25h5+xtSyhPF4s3TMzCpx4GdErUsY9/f1svW2f9ngqsyskzjwM6DWJYx7J4nb53+gwVWZWafxGH4GrF6/veqw7+/rZfWn/bSsmdXOPfwMqGY6pcfpzSwpB34GHGuxMy9hbGZp8ZBOBiy99Gz6enuOet1LGJtZmtzDz4DRQF+9fjt79o8w3cM3ZtYADvwGGp1TX02Ie7EzM2s0B36DrMgP88Cmlxl9ULbSipZmZs2UaAxf0hRJGyTtKP18SoU275P0Y0lbJW2TdFOSa7aD/FDhiLAfNbqipZlZKyS9absM2BgRs4GNpfOx9gIfjIjzgTnAMknTE14301av335U2I/yipZm1ipJA38BcH/p+H7g8rENIuLtiHirdHp8CtfMvGOFule0NLNWSRq+0yJib+n4Z8C0So0kzZT0LLALuDsi9ozTbrGkQUmD+/btS1ha64wX6gKvaGlmLTNh4Et6QtJPKvxYUN4uiruhVxzJiIhdEXEe8H5gkaSKfzFExJqIyEVEburUqXX852RDpXn1AhZeNOAbtmbWMhPO0omIueO9J+kVSadHxF5JpwOvTvBdeyT9BPgQ8GjN1bYJz6s3syxKOi1zLbAIWFX6+TtjG0g6A3gtIkZKs3j+CPhiwutmnufVm1nWJB3DXwXMk7QDmFs6R1JO0n2lNv8c2CzpGeB/Af85IoYTXtfMzGqUqIcfEa8BH6vw+iBwY+l4A3BekuuYmVlyHT9F0szMihz4ZmZdwoFvZtYlHPhmZl3CgW9m1iUc+GZmXaIr18OvZWMSM7NO0XWBnx8qcMtjw4wcOAR4YxIz6x5dN6Szev32w2E/yhuTmFk36OgefqWhm/HWqvfGJGbW6To28Mcbunl3Xy/7Rw4c1d4bk5hZp+vYIZ3xhm4kjlqrvq+3xxuTmFnH69jAH2+IZv+bB1h5xbnM6O9DwIz+PlZeca5v2JpZx+vYIZ3p/X0UKoT+9P4+r1VvZl2pY3v4lbYZ9NCNmXWzju3he5tBM7MjJQp8SVOAh4BZwIvAVRHx+jhtTwaeA/IRcXOS61bLQzdmZr+RdEhnGbAxImYDG0vn4/lr4H8nvJ6ZmdUpaeAvAO4vHd8PXF6pkaQ/AKYB/zPh9czMrE5JA39aROwtHf+MYqgfQdIk4G+Av5royyQtljQoaXDfvn0JSzMzs3ITjuFLegJ4b4W3lpefRERIigrtPgusi4jdko55rYhYA6wByOVylb7LzMzqNGHgR8Tc8d6T9Iqk0yNir6TTgVcrNPsg8CFJnwXeBUyW9P8i4ljj/WZmlrKk0zLXAouAVaWfvzO2QUQsHD2WdD2Qc9ibmTVf0jH8VcA8STuAuaVzJOUk3Ze0ODMzS48isjlUnsvlYnBwsNVlmJm1FUlPR0Su0nsdu7SCmZkdKbM9fEn7gJcmaHYq8E9NKKcerq0+Wa0tq3WBa6tXVmtLWtf7ImJqpTcyG/jVkDQ43j9dWs211SertWW1LnBt9cpqbY2sy0M6ZmZdwoFvZtYl2j3w17S6gGNwbfXJam1ZrQtcW72yWlvD6mrrMXwzM6teu/fwzcysSg58M7Mu0VaBL2mKpA2SdpR+PuUYbU+WtFvSl7NSm6T3SfqxpK2Stkm6KUO1nS/pqVJdz0q6Oiu1ldp9T9J+SX/X4Houk7Rd0k5JR635JOl4SQ+V3t8saVYj66mxtg+X/nwdlHRls+qqsrYlkp4r/dnaKOl9GanrJknDpf8nfyjpnGbUVU1tZe0+JSkkJZ+qGRFt8wP4ArCsdLwMuPsYbb8EfBP4clZqAyYDx5eO30VxW8jpGantnwGzS8fTgb1AfxZqK733MeCTwN81sJYe4Hngt0u/V88A54xp81ngv5aOrwEeatKfr2pqmwWcB3wduLIZddVQ278ATigd/1kzft2qrOvksuP5wPey8mtWancSxZ0CN1FceDLRdduqh0+2d9iasLaIeDsi3iqdHk/z/oVVTW0/jYgdpeM9FJe6rvi0XrNrK9W0EXijwbVcCOyMiBci4m3gwVJ95crrfRT4mCba6KFJtUXEixHxLPBOE+qptbYfRMSbpdNNwBkZqeuXZacnAs2axVLNnzUobg17N/DrNC7aboGf6g5bKZuwNgBJMyU9C+yi2Jvdk5XaRkm6kGKv4/lGF0aNtTXYDIq/L6N2l16r2CYiDgK/AN6TkdpapdbabgD+vqEVFVVVl6Q/l/Q8xX9t/rsm1FVVbZJ+H5gZEY+nddGk6+Gnrpk7bLWgNiJiF3CepOlAXtKjEfFKFmorfc/pwDeARRGRSk8xrdqs/Um6DsgBH2l1LaMi4l7gXkmfAVZQ3NujpUod13uA69P83swFfmR4h60Uaiv/rj2SfgJ8iOLQQMtrk3Qy8DiwPCI2Ja0pzdqapADMLDs/o/RapTa7JR0HvBt4LSO1tUpVtUmaS/Ev+Y+UDW22vK4yDwJ/29CKfmOi2k4Cfhd4stRxfS+wVtL8iKh73fh2G9IZ3WELjrHDVkQMRMQsisM6X08j7NOoTdIZkvpKx6cAfwRsz0htk4FvU/z1SvwXUJq1NdEWYLakM0u/HtdQrK9ceb1XAt+P0t21DNTWKhPWJukC4CvA/Iho1l/q1dQ1u+z0E8COLNQWEb+IiFMjYlYpyzZR/LVLtklIM+5Ip/WD4ljpRoq/KU8AU0qv54D7KrS/nubN0pmwNmAe8CzFO/LPAoszVNt1wAFga9mP87NQW+n8H4B9wAjF8c5LG1TPx4GfUrx/sbz02p2l/9kAfgt4BNgJ/B/gt5vxe1hlbX9Y+rX5FcV/dWzLUG1PAK+U/dlam5G6vgRsK9X0A+ADWfk1G9P2SVKYpeOlFczMukS7DemYmVmdHPhmZl3CgW9m1iUc+GZmXcKBb2bWJRz4ZmZdwoFvZtYl/j8hT4+lVtUaEwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x=np.reshape(interactions, -1),\n",
    "            y=np.reshape(orig_interactions_summed, -1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
